import java.io.*;
import java.net.*;
import java.util.*;
import org.xml.sax.*;
import org.xml.sax.helpers.XMLReaderFactory;

class RoarHarvester {
	public static void main (String[] args) {
		RoarHarvester rh = new RoarHarvester(args);
	}
	
	public RoarHarvester (String[] args) {
		String file = "test.html";
		int records = 10;
		System.out.println("Making sub directory for " + file);
		mkSubDir(file,records);
	}
	
	public void mkSubDir(String file, int records) {
		Vector data = getContent(file);
		Random generator = new Random();
		int size = data.size();
		Hashtable done = new Hashtable();
		for (int i=0;i<records;i++) {
			int n = generator.nextInt(size);
			try {
				while (done.get(n).equals("true")) {
					n = generator.nextInt(size);
				}
			} catch (Exception e) {
			}
			done.put(n,"true");
			String line = (String)data.get(n);
			String[] bits = line.split("\t");
			String file_url = bits[1];
			System.out.println(i + " Downloading : " + file_url);
			download(file_url);
		}
	}

	public Vector getContent(String file) {
		try {
			XMLReader parser = XMLReaderFactory.createXMLReader();
			parser.setContentHandler(new XMLContentHandler());
			parser.parse(file);
		} catch (Exception e) {
			e.printStackTrace();
		}
		
		Vector v = new Vector();
		
		try {
			BufferedReader in = new BufferedReader(new FileReader("test-out.txt"));
			String str;
			while ((str = in.readLine()) != null) {
				v.add(str);
			}
			in.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		
		return v;
	}
	
	public static void download(String address, String localFileName) {
		OutputStream out = null;
		URLConnection conn = null;
		InputStream  in = null;
		try {
			URL url = new URL(address);
			out = new BufferedOutputStream(
					new FileOutputStream(localFileName));
			conn = url.openConnection();
			in = conn.getInputStream();
			byte[] buffer = new byte[1024];
			int numRead;
			long numWritten = 0;
			while ((numRead = in.read(buffer)) != -1) {
				out.write(buffer, 0, numRead);
				numWritten += numRead;
			}
			System.out.println(localFileName + "\t" + numWritten);
		} catch (Exception exception) {
			exception.printStackTrace();
		} finally {
			try {
				if (in != null) {
					in.close();
				}
				if (out != null) {
					out.close();
				}
			} catch (IOException ioe) {
			}
		}
	}

	public static void download(String address) {
		int lastSlashIndex = address.lastIndexOf('/');
		if (lastSlashIndex >= 0 &&
				lastSlashIndex < address.length() - 1) {
			download(address, "out/" + address.substring(lastSlashIndex + 1));
		} else {
			System.err.println("Could not figure out local file name for " +
					address);
		}
	}

}
